/**
 * @author 李亚松
 * @email  lysong01@gmail.com
 * 
 * AbstractCrawler.java 2012-12-2
 * description 
 */
package liyasong.tool.crawler;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import liyasong.tool.crawler.beans.ArticleBean;

public abstract class AbstractCrawler implements Crawler {

	private String blogSite;
	
	private String charset = "utf-8";
	
	private List<ArticleBean> articles;

	public AbstractCrawler(String blogSite) {
		this.blogSite = blogSite;
		articles = new ArrayList<>();
	}

	public AbstractCrawler(String blogSite, String charset) {
		this.blogSite = blogSite;
		this.charset = charset;
		articles = new ArrayList<>();
	}

	public List<ArticleBean> crawl() {
		System.out.println("crawler start!");
		try {
			URL urlPage = new URL(blogSite);
			HttpURLConnection conn = (HttpURLConnection) urlPage.openConnection();	 
			Parser parser = new Parser(conn);
			parser.setEncoding(charset);
			
			NodeList aNodeList = parser.parse(authorFilter());
			String author = aNodeList.elementAt(0).toPlainTextString();
			author = cleanAuthor(author);
			
			parser.reset();
			NodeList fNodeList = parser.parse(firstLinkFilter());
			LinkTag firstLink = (LinkTag) fNodeList.elementAt(0);
			
			String blogLink = firstLink.getLink();
			while (blogLink != null) {
				parser = new Parser(blogLink);
				String title = parser.parse(titleFilter()).elementAt(0).toPlainTextString();
				parser.reset();
				String description = parser.parse(descriptionFilter()).elementAt(0).toHtml();
				parser.reset();
				String pubDate = parser.parse(pubDateFilter()).elementAt(0).toPlainTextString();
				String link = blogLink;

				ArticleBean article = new ArticleBean();
				article.setAuthor(author);
				article.setTitle(cleanTitle(title));
				article.setDescription(cleanDescription(description));
				article.setPubDate(pubDate);
				article.setLink(link);
				articles.add(article);
				
				System.out.println("crawl the article:"+title);
				
				parser.reset();
				NodeList lNodeList = parser.parse(lastLinkFilter());
				if (lNodeList.size() != 0) {
					LinkTag lastLink = (LinkTag)(lNodeList.elementAt(0));
					blogLink = lastLink.getLink(); 
				} else {
					blogLink = null;
				}
//				break;
			}
			
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ParserException e) {
			e.printStackTrace();
		}
		System.out.println("crawl stop!");
		return articles;
	}

	protected abstract NodeFilter firstLinkFilter();
	protected abstract NodeFilter lastLinkFilter();
	protected abstract NodeFilter titleFilter();
	protected abstract NodeFilter authorFilter();
	protected abstract NodeFilter descriptionFilter();
	protected abstract NodeFilter pubDateFilter();
	
	protected abstract String cleanAuthor(String author);
	protected abstract String cleanTitle(String title);
	protected abstract String cleanDescription(String description);

	/* (non-Javadoc)
	 * @see liyasong.tool.crawler.Crawler#getArticles()
	 */
	@Override
	public List<ArticleBean> getArticles() {
		return articles;
	}

	@Override
	public void resultWrite(String outPath) {
		System.out.println("articles num: " + articles.size());
		String osName = System.getProperty("os.name");
		String dirSeparator = osName.startsWith("Win") ? "\\":"/";
		for (ArticleBean article : articles) {
			System.out.println("begin to write:"+article.getTitle());
			String dirName = article.getAuthor();
			String path	= outPath + dirSeparator + dirName;
			File dir = new File(path);
			if (!dir.exists()) {
				dir.mkdir();
			}
			File file = new File(path + dirSeparator + article.getTitle() + ".txt");
			try {
				FileWriter fw = new FileWriter(file);
				String description = article.getDescription();
				fw.write(description);
				fw.flush();
				fw.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

}
